from config.url import *
from pyquery.pyquery import PyQuery as pyq
from utils.WriteFile import write_json
from utils.ReadFile import read_json
from utils.Common import remove_repeat


def get(page):
    results = []
    dj_url = URL['dajie'] + str(page)
    doc = pyq(dj_url)
    content = doc('.job-name a')
    for i in content.items():
        results.append(i.text())
        pass
    return results
    pass


def get_all(max_page):
    p = '../data/company.json'
    rs = read_json(p)
    for i in range(1, max_page):
        tmp = get(i)
        rs.extend(tmp)
        print(tmp)
        pass
    rs = remove_repeat(rs)
    write_json(rs, p)
    pass
